# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem
from scrapy.conf import settings   #导入settings 是为了在pipeline中调用setting中定义的各DB参数
import scrapy
import codecs
import json
import pymysql
import pymongo
#通过以下pipeline使用pymysql连接并写入mysql server
class BookinfoStoreMysqlPipeline(object):
    def __init__(self):
        pass
    def dbHandle(self):
        conn = pymysql.connect(host=settings.get('MYSQL_HOST'), db=settings.get('MYSQL_DBNAME'),
                               user=settings.get('MYSQL_USER'), passwd=settings.get('MYSQL_PASSWD'),
                               charset='utf-8')
        return conn
    def process_item(self,item,spider):
        conn=self.dbHandle()
        cursor=conn.cursor()
        insert_sql = 'insert into bookinfo(name,author,publisher,url) VALUES (%s,%s,%s,%s)'
        try:
            cursor.execute(insert_sql,(item["name"],item["author"],item["publisher"],item["image_urls"]))
            conn.commit()
        except:
            conn.rollback()
        conn.close()
        return item
    def spider_close(self,spider):
        pass
#通过以下pipeline使用json.dumps形式将返回的数据写入指定json文件中
class BookinfoStoreFilePipeline(object):
    def __init__(self):
        self.file=codecs.open('bookinfo.json','w',encoding='utf-8')
    def process_item(self,item,spider):
        line = json.dumps(dict(item),ensure_ascii=False) + "\n"
        self.file.write(line)
        return item
    def spider_colse(self,spider):
        self.file.close()
#通过以下pipeline使用pymongo形式将返回的数据写入mongodb中
class BookinfoStoreMongoPipeline(object):
    def __init__(self):
        conn=pymongo.MongoClient(settings['MONGODB_HOST'],settings['MONGODB_PORT'])
        db=conn[settings['MONGODB_DB']]
        self.collection = db[settings['MONGODB_COLLECTION']]
    def process_item(self,item,spider):
        self.collection.insert(dict(item))
        return item
    def spider_colse(self,spider):
        self.conn.close()
#通过以下pipeline下载抓取过程中存储在image_urls字段的图片
class BookImgsDLPipeline(ImagesPipeline):
    default_headers = {
        'accept': 'image/webp,image/*,*/*;q=0.8',
        'accept-encoding': 'gzip, deflate, sdch, br',
        'accept-language': 'zh-CN,zh;q=0.8,en;q=0.6',
        'referer': 'https://book.douban.com/top250/',
        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
    }
    def get_media_requests(self,item,info):
        self.default_headers['referer'] = item['image_urls']
        yield scrapy.Request(item['image_urls'], headers=self.default_headers, meta={'bookname':item['name']})
        #将每个bookname附加在meta中传递给下一个函数处理
    def file_path(self,request,response=None,info=None):
        bookname=request.meta['bookname']
        image_guid = bookname+'_'+request.url.split('/')[-1]  #自定义保存图片的名称
        filename = 'full/%s' % (image_guid)
        return filename
    def item_completed(self,results,item,info):
        image_paths = [ value['path'] for ok, value in results if ok ]
        if not image_paths:
            raise DropItem("Item contains no images")
        item['image_paths'] = image_paths[0]
        item['images'] = [ value for ok, value in results if ok ]
        return item
